import org.apache.spark.sql.SparkSession
object ag {
  def main(args: Array[String]): Unit = {
    // 创建 SparkSession
    val spark = SparkSession.builder()
      .appName("WordCount")
      .master("local[*]")
      .getOrCreate()
    val sc = spark.sparkContext
    try {
      val first_half = sc.textFile("file:///D:/Employee_salary_first_half.csv")
      val second_half = sc.textFile("file:///D:/Employee_salary_second_half.csv")
      val drop_first = first_half.mapPartitionsWithIndex((ix, it) => {
        if (ix == 0) it.drop(1)
        it
      })
      val drop_second = second_half.mapPartitionsWithIndex((ix, it) => {
        if (ix == 0) it.drop(1)
        it
      })
      val split_first = drop_first.map(line => {
        val data = line.split(",");
        (data(1), data(6).toInt)
      })
      val split_second = drop_second.map(line => {
        val data = line.split(",");
        (data(1), data(6).toInt)
      })

    } finally {
      // 停止 SparkSession
      spark.stop()
    }
  }
}
